#function used to scrap the data

library(tidyverse)
library(rvest)

get_review <- function(url, n) {
  review <- data.frame()
  
  if (n %% 5 != 0) {stop("n need to be a multiple of 5")}
  
  number_pages <- n/5
  
  for (i in seq_len(number_pages)) {
    blocks_url <- str_split(url, "-") %>% unlist()
    start_url <- paste0(blocks_url[1], "-", blocks_url[2], "-", blocks_url[3], "-", blocks_url[4])
    end_url <- paste0("-", blocks_url[5])
    
    if (i == 1) {
      review_url <- paste0(start_url, end_url, "#REVIEWS")
    }
    
    else {
      review_url <- paste0(start_url, "-or", (i-1)*5, end_url, "#REVIEWS")
    }
    
    html <- read_html(review_url)
    
    for(r in 3:7){
    
    temp <- data_frame()
    temp_rev <- data_frame()
    temp_date <- data_frame()
    temp_rating <- data_frame()
    
    revi <- html_nodes(html, paste0('#component_15 > div > div:nth-child(3) > div:nth-child(',r,') > div.oETBfkHU > div._3hDPbqWO > div._2f_ruteS._1bona3Pu > div.cPQsENeY > q'))
    temp_rev <- html_text(revi)
    
    date_r <- html_nodes(html, paste0('#component_15 > div > div:nth-child(3) > div:nth-child(',r,') > div.oETBfkHU > div._3hDPbqWO > div._1O8E5N17 > span._34Xs-BQm'))
    temp_date <- html_text(date_r) %>% str_sub(start= 15)
    
    temp <- temp_rev %>% merge(temp_date)
    
    rating_r <- html_nodes(html, paste0('#component_15 > div > div:nth-child(3) > div:nth-child(',r,') > div.oETBfkHU > div._2UEC-y30 > div > span'))
    temp_rating <- rating_r %>% xml_attr("class") %>% str_sub(start = -2) %>% as.numeric()/10
    
    temp <- temp %>% cbind(temp_rating)
    
    colnames(temp) <- c("review", "date_stay", "rating")
    
    review <- review %>% rbind(temp)
    }
  }
  return(review)
}


#For our specific use

# urls of the hotels
url <- c(
  "https://www.tripadvisor.com/Hotel_Review-g60763-d93359-Reviews-Econo_Lodge_Times_Square-New_York_City_New_York.html",
  "https://www.tripadvisor.com/Hotel_Review-g60763-d290978-Reviews-Hotel_St_James-New_York_City_New_York.html",
  "https://www.tripadvisor.com/Hotel_Review-g60763-d10330604-Reviews-Four_Seasons_Hotel_New_York_Downtown-New_York_City_New_York.html",
  "https://www.tripadvisor.com/Hotel_Review-g60763-d113311-Reviews-The_Peninsula_New_York-New_York_City_New_York.html"
)

# get 50 reviews per hotel
n <- 50

econo_review <- get_review(url[1], n)
st_james_review <- get_review(url[2], n)
four_seasons_review <- get_review(url[3], n)
peninsula_review <- get_review(url[4], n)

#save environment 

save(econo_review, st_james_review, four_seasons_review, peninsula_review, file = here::here("Data/data_scrap_50.RData"))


# Get all reviews
n <- 500

econo_review <- get_review(url[1], n)
st_james_review <- get_review(url[2], n)
four_seasons_review <- get_review(url[3], n)
peninsula_review <- get_review(url[4], n)

# merge and save reviews

econo_review_ <- econo_review %>% mutate(hotel = "Econo")
st_james_review_ <- st_james_review %>% mutate(hotel = "St-james")
four_seasons_review_ <- four_seasons_review %>% mutate(hotel = "four seasons")
peninsula_review_ <- peninsula_review %>% mutate(hotel = "Peninsula")

reviews <- rbind(econo_review_, st_james_review_, four_seasons_review_, peninsula_review_)
saveRDS(reviews, file = here::here("Data/Reviews_500.rds"))

#load data from overall_scrap
load(file = here::here("Data/data_scrap_50.RData"))

# To begin the analysis, we convert all reviews into "character" type.
econo_review$review <- as.character(econo_review$review) 
four_seasons_review$review <- as.character(four_seasons_review$review)
peninsula_review$review <- as.character(peninsula_review$review)
st_james_review$review <- as.character(st_james_review$review)


## Create corpus
#Econo Lodge
econo_reviews.tb <- as_tibble(data.frame(econo_review))
econo_reviews.cp <- corpus(econo_review$review)
#summary(econo_reviews.cp)

#Four Seasons
four_seasons_reviews.tb <- as_tibble(data.frame(four_seasons_review))
four_seasons_reviews.cp <- corpus(four_seasons_review$review)
#summary(fseasons_reviews.cp)

#Peninsula
peninsula_reviews.tb <- as_tibble(data.frame(peninsula_review))
peninsula_reviews.cp <- corpus(peninsula_review$review)
#summary(peninsula_reviews.cp)

#St-James
st_james_reviews.tb <- as_tibble(data.frame(st_james_review))
st_james_reviews.cp <- corpus(st_james_review$review)
#summary(st_james_reviews.cp)

#simplifier avec un for loop?

## Tokenization

#Econo Lodge
econo_reviews.tok <- tokens(econo_reviews.cp, remove_numbers=TRUE, remove_punct=TRUE, remove_symbols=TRUE, remove_separators=TRUE)

econo_reviews.tok <- econo_reviews.tok %>% tokens_tolower() %>% tokens_replace(pattern=hash_lemmas$token, replacement = hash_lemmas$lemma) %>% tokens_remove(stopwords("english"))

#Four Seasons
four_seasons_reviews.tok <- tokens(four_seasons_reviews.cp, remove_numbers=TRUE, remove_punct=TRUE, remove_symbols=TRUE, remove_separators=TRUE)

four_seasons_reviews.tok <-four_seasons_reviews.tok %>% tokens_tolower() %>% tokens_replace(pattern=hash_lemmas$token, replacement = hash_lemmas$lemma) %>% tokens_remove(stopwords("english"))

#Peninsula
peninsula_reviews.tok <- tokens(peninsula_reviews.cp, remove_numbers=TRUE, remove_punct=TRUE, remove_symbols=TRUE, remove_separators=TRUE)

peninsula_reviews.tok <-peninsula_reviews.tok %>% tokens_tolower() %>% tokens_replace(pattern=hash_lemmas$token, replacement = hash_lemmas$lemma) %>% tokens_remove(stopwords("english"))

#St-James
st_james_reviews.tok <- tokens(st_james_reviews.cp, remove_numbers=TRUE, remove_punct=TRUE, remove_symbols=TRUE, remove_separators=TRUE)

st_james_reviews.tok <-st_james_reviews.tok %>% tokens_tolower() %>% tokens_replace(pattern=hash_lemmas$token, replacement = hash_lemmas$lemma) %>% tokens_remove(stopwords("english"))